Cargando paquetes

Rutas


raw_data <- here("data", "raw")
interim_data <- here("data", "interim")
processed_data <- here("data", "processed")

Leyendo bases

base <- readRDS(file=paste0(processed_data,"/base_final.Rds"))

base <- base %>%
  mutate(objetivo = if_else(objetivo==0, "bueno", "malo"))

head(base)

Tarea de aprendizaje

Definicion

task_tarj = TaskClassif$new(id = "tarjetas", backend = base, target = "objetivo")
Error: Target column 'objetivo' must have at least two levels

autoplot(task_tarj$select(c("saldo_tarjeta", "coeficiente_solvencia", "edad", "limite_tarjeta_credito")), type = "pairs")

task_tarj = TaskClassif$new(id = "tarjetas", backend = base, target = "buen_pagador")

Modelos

Lista de modelos


lrn_rpart = lrn("classif.rpart", predict_type = "prob", predict_sets = c("train", "test"))
lrn_glmnet = lrn("classif.glmnet", predict_type = "prob", predict_sets = c("train", "test"))
lrn_knn = lrn("classif.kknn", predict_type = "prob", predict_sets = c("train", "test"))
lrn_lda = lrn("classif.lda", predict_type = "prob", predict_sets = c("train", "test"))
lrn_log_reg = lrn("classif.log_reg", predict_type = "prob", predict_sets = c("train", "test"))
lrn_bayes = lrn("classif.naive_bayes", predict_type = "prob", predict_sets = c("train", "test"))
lrn_rf = lrn("classif.ranger", predict_type = "prob", predict_sets = c("train", "test"))
lrn_svm = lrn("classif.svm", predict_type = "prob", predict_sets = c("train", "test"))
lrn_xgboost = lrn("classif.xgboost", predict_type = "prob", predict_sets = c("train", "test"))

Separacion de training y testing

set.seed(34678)
train_set = sample(task_tarj$nrow, 0.8 * task_tarj$nrow)
test_set = setdiff(seq_len(task_tarj$nrow), train_set)

Benchmark

Prepracion de datos para el benchmark


impute_fcts <- po("imputemode", affect_columns = selector_type("factor"))
impute_nums <- po("imputehist", affect_columns = selector_type("numeric"))
encode <- po("encode", affect_columns = selector_type("factor"))

pre_procesamiento <- impute_fcts %>>% 
  impute_nums %>>% 
  encode

lrn_rpart <- GraphLearner$new(pre_procesamiento %>>% po(lrn_rpart))
lrn_glmnet <- GraphLearner$new(pre_procesamiento %>>% po(lrn_glmnet))
lrn_knn <- GraphLearner$new(pre_procesamiento %>>% po(lrn_knn))
lrn_lda <- GraphLearner$new(pre_procesamiento %>>% po(lrn_lda))
lrn_log_reg <- GraphLearner$new(pre_procesamiento %>>% po(lrn_log_reg))
lrn_bayes <- GraphLearner$new(pre_procesamiento %>>% po(lrn_bayes))
lrn_rf <- GraphLearner$new(pre_procesamiento %>>% po(lrn_rf))
lrn_svm <- GraphLearner$new(pre_procesamiento %>>% po(lrn_svm))
lrn_xgboost <- GraphLearner$new(pre_procesamiento %>>% po(lrn_xgboost))

lrn_rpart$predict_sets = c("train", "test")
lrn_glmnet$predict_sets = c("train", "test")
lrn_knn$predict_sets = c("train", "test")
lrn_lda$predict_sets = c("train", "test")
lrn_log_reg$predict_sets = c("train", "test")
lrn_bayes$predict_sets = c("train", "test")
lrn_rf$predict_sets = c("train", "test")
lrn_svm$predict_sets = c("train", "test")
lrn_xgboost$predict_sets = c("train", "test")

lrn_rpart$id = "Árbol"
lrn_glmnet$id = "Reg-reg"
lrn_knn$id = "K-vecinos"
lrn_lda$id = "LDA"
lrn_log_reg$id = "Reg-log"
lrn_bayes$id = "Bayes"
lrn_rf$id = "RandomForest"
lrn_svm$id = "SVM"
lrn_xgboost$id = "XGBoost"

learners = list(
  lrn_rpart,
  lrn_glmnet,
  lrn_knn,
  lrn_lda,
  lrn_log_reg,
  lrn_bayes,
  lrn_rf,
  lrn_svm,
  lrn_xgboost
)

Defincion del benchmark

resamplings = rsmp("cv", folds = 10)

design = benchmark_grid(task_tarj, learners, resamplings)

Ejecucion

future::plan("multiprocess")
[ONE-TIME WARNING] Forked processing ('multicore') is disabled in future (>= 1.13.0) when running R from RStudio, because it is considered unstable. Because of this, plan("multicore") will fall back to plan("sequential"), and plan("multiprocess") will fall back to plan("multisession") - not plan("multicore") as in the past. For more details, how to control forked processing or not, and how to silence this warning in future R sessions, see ?future::supportsMulticore
bmr = benchmark(design)
INFO  [13:51:43.131] Benchmark with 90 resampling iterations 
INFO  [13:51:47.851] Applying learner 'SVM' on task 'tarjetas' (iter 5/10) 
INFO  [14:03:56.824] Applying learner 'XGBoost' on task 'tarjetas' (iter 9/10) 
INFO  [14:04:12.361] Applying learner 'RandomForest' on task 'tarjetas' (iter 1/10) 
INFO  [14:05:02.021] Applying learner 'Reg-reg' on task 'tarjetas' (iter 8/10) 
INFO  [14:05:09.841] Applying learner 'LDA' on task 'tarjetas' (iter 4/10) 
INFO  [14:05:15.472] Applying learner 'XGBoost' on task 'tarjetas' (iter 5/10) 
INFO  [14:05:22.733] Applying learner 'SVM' on task 'tarjetas' (iter 2/10) 
INFO  [14:09:07.831] Applying learner 'LDA' on task 'tarjetas' (iter 9/10) 
INFO  [14:09:09.914] Applying learner 'RandomForest' on task 'tarjetas' (iter 7/10) 
INFO  [14:09:25.559] Applying learner 'Reg-reg' on task 'tarjetas' (iter 5/10) 
INFO  [14:09:27.877] Applying learner 'K-vecinos' on task 'tarjetas' (iter 9/10) 
INFO  [14:09:53.303] Applying learner 'Árbol' on task 'tarjetas' (iter 6/10) 
INFO  [14:09:55.347] Applying learner 'LDA' on task 'tarjetas' (iter 10/10) 
INFO  [14:09:57.423] Applying learner 'LDA' on task 'tarjetas' (iter 1/10) 
INFO  [14:09:59.604] Applying learner 'RandomForest' on task 'tarjetas' (iter 5/10) 
INFO  [14:10:16.323] Applying learner 'Reg-log' on task 'tarjetas' (iter 6/10) 
INFO  [14:10:20.691] Applying learner 'Reg-log' on task 'tarjetas' (iter 7/10) 
INFO  [14:10:24.748] Applying learner 'LDA' on task 'tarjetas' (iter 2/10) 
INFO  [14:10:26.956] Applying learner 'Árbol' on task 'tarjetas' (iter 10/10) 
INFO  [14:10:29.972] Applying learner 'Árbol' on task 'tarjetas' (iter 8/10) 
INFO  [14:10:32.253] Applying learner 'SVM' on task 'tarjetas' (iter 10/10) 
INFO  [14:18:40.391] Applying learner 'LDA' on task 'tarjetas' (iter 8/10) 
INFO  [14:18:44.308] Applying learner 'RandomForest' on task 'tarjetas' (iter 2/10) 
variables are collinearvariables are collinearvariables are collinearvariables are collinearprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingvariables are collinearvariables are collinear
INFO  [13:51:53.174] Applying learner 'LDA' on task 'tarjetas' (iter 7/10) 
INFO  [13:52:03.373] Applying learner 'K-vecinos' on task 'tarjetas' (iter 4/10) 
INFO  [13:53:19.560] Applying learner 'Reg-reg' on task 'tarjetas' (iter 3/10) 
INFO  [13:53:30.434] Applying learner 'RandomForest' on task 'tarjetas' (iter 4/10) 
INFO  [13:54:20.423] Applying learner 'SVM' on task 'tarjetas' (iter 8/10) 
INFO  [14:05:30.140] Applying learner 'LDA' on task 'tarjetas' (iter 6/10) 
INFO  [14:05:32.712] Applying learner 'K-vecinos' on task 'tarjetas' (iter 7/10) 
INFO  [14:05:55.551] Applying learner 'K-vecinos' on task 'tarjetas' (iter 8/10) 
INFO  [14:06:21.490] Applying learner 'Árbol' on task 'tarjetas' (iter 4/10) 
INFO  [14:06:24.828] Applying learner 'XGBoost' on task 'tarjetas' (iter 10/10) 
INFO  [14:06:28.969] Applying learner 'Reg-reg' on task 'tarjetas' (iter 1/10) 
INFO  [14:06:31.656] Applying learner 'Bayes' on task 'tarjetas' (iter 9/10) 
INFO  [14:06:53.035] Applying learner 'XGBoost' on task 'tarjetas' (iter 4/10) 
INFO  [14:06:55.223] Applying learner 'Bayes' on task 'tarjetas' (iter 4/10) 
INFO  [14:07:15.253] Applying learner 'SVM' on task 'tarjetas' (iter 3/10) 
INFO  [14:11:35.424] Applying learner 'Bayes' on task 'tarjetas' (iter 7/10) 
INFO  [14:12:34.694] Applying learner 'K-vecinos' on task 'tarjetas' (iter 10/10) 
INFO  [14:13:50.922] Applying learner 'Reg-reg' on task 'tarjetas' (iter 10/10) 
INFO  [14:13:58.568] Applying learner 'RandomForest' on task 'tarjetas' (iter 3/10) 
INFO  [14:14:43.545] Applying learner 'XGBoost' on task 'tarjetas' (iter 6/10) 
INFO  [14:14:48.297] Applying learner 'Reg-reg' on task 'tarjetas' (iter 7/10) 
INFO  [14:14:55.788] Applying learner 'XGBoost' on task 'tarjetas' (iter 7/10) 
variables are collinearvariables are collinear
INFO  [13:51:58.165] Applying learner 'Reg-log' on task 'tarjetas' (iter 3/10) 
INFO  [13:52:10.773] Applying learner 'K-vecinos' on task 'tarjetas' (iter 6/10) 
INFO  [13:53:35.219] Applying learner 'Árbol' on task 'tarjetas' (iter 5/10) 
INFO  [13:53:44.797] Applying learner 'SVM' on task 'tarjetas' (iter 6/10) 
INFO  [14:05:35.308] Applying learner 'Árbol' on task 'tarjetas' (iter 3/10) 
INFO  [14:05:37.897] Applying learner 'SVM' on task 'tarjetas' (iter 1/10) 
INFO  [14:09:07.735] Applying learner 'Reg-log' on task 'tarjetas' (iter 10/10) 
INFO  [14:09:10.470] Applying learner 'Bayes' on task 'tarjetas' (iter 2/10) 
INFO  [14:09:34.768] Applying learner 'Reg-log' on task 'tarjetas' (iter 1/10) 
INFO  [14:09:37.510] Applying learner 'Árbol' on task 'tarjetas' (iter 1/10) 
INFO  [14:09:39.897] Applying learner 'K-vecinos' on task 'tarjetas' (iter 2/10) 
INFO  [14:10:10.328] Applying learner 'Árbol' on task 'tarjetas' (iter 9/10) 
INFO  [14:10:12.726] Applying learner 'RandomForest' on task 'tarjetas' (iter 6/10) 
INFO  [14:10:30.708] Applying learner 'Bayes' on task 'tarjetas' (iter 1/10) 
INFO  [14:10:52.541] Applying learner 'SVM' on task 'tarjetas' (iter 7/10) 
INFO  [14:19:23.859] Applying learner 'Bayes' on task 'tarjetas' (iter 8/10) 
INFO  [14:19:53.067] Applying learner 'Reg-reg' on task 'tarjetas' (iter 9/10) 
INFO  [14:19:57.209] Applying learner 'RandomForest' on task 'tarjetas' (iter 8/10) 
INFO  [14:20:22.666] Applying learner 'LDA' on task 'tarjetas' (iter 5/10) 
INFO  [14:20:26.858] Applying learner 'Reg-log' on task 'tarjetas' (iter 9/10) 
INFO  [14:20:31.870] Applying learner 'K-vecinos' on task 'tarjetas' (iter 1/10) 
INFO  [14:21:03.157] Applying learner 'Reg-log' on task 'tarjetas' (iter 5/10) 
prediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingvariables are collinearprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleading
INFO  [13:52:07.032] Applying learner 'Bayes' on task 'tarjetas' (iter 3/10) 
INFO  [13:53:20.128] Applying learner 'K-vecinos' on task 'tarjetas' (iter 3/10) 
INFO  [13:54:54.520] Applying learner 'Árbol' on task 'tarjetas' (iter 7/10) 
INFO  [13:55:02.302] Applying learner 'Reg-reg' on task 'tarjetas' (iter 6/10) 
INFO  [13:55:14.226] Applying learner 'RandomForest' on task 'tarjetas' (iter 10/10) 
INFO  [13:56:02.095] Applying learner 'Bayes' on task 'tarjetas' (iter 6/10) 
INFO  [13:57:16.564] Applying learner 'SVM' on task 'tarjetas' (iter 4/10) 
INFO  [14:06:19.630] Applying learner 'RandomForest' on task 'tarjetas' (iter 9/10) 
INFO  [14:06:35.774] Applying learner 'XGBoost' on task 'tarjetas' (iter 3/10) 
INFO  [14:06:38.084] Applying learner 'Árbol' on task 'tarjetas' (iter 2/10) 
INFO  [14:06:40.168] Applying learner 'K-vecinos' on task 'tarjetas' (iter 5/10) 
INFO  [14:07:04.088] Applying learner 'XGBoost' on task 'tarjetas' (iter 8/10) 
INFO  [14:07:06.284] Applying learner 'Reg-log' on task 'tarjetas' (iter 2/10) 
INFO  [14:07:08.776] Applying learner 'SVM' on task 'tarjetas' (iter 9/10) 
INFO  [14:11:32.237] Applying learner 'LDA' on task 'tarjetas' (iter 3/10) 
INFO  [14:11:39.076] Applying learner 'XGBoost' on task 'tarjetas' (iter 2/10) 
INFO  [14:11:45.433] Applying learner 'Reg-reg' on task 'tarjetas' (iter 2/10) 
INFO  [14:11:54.250] Applying learner 'Reg-reg' on task 'tarjetas' (iter 4/10) 
INFO  [14:12:00.431] Applying learner 'Reg-log' on task 'tarjetas' (iter 4/10) 
INFO  [14:12:07.894] Applying learner 'Bayes' on task 'tarjetas' (iter 10/10) 
INFO  [14:13:10.240] Applying learner 'Reg-log' on task 'tarjetas' (iter 8/10) 
INFO  [14:13:20.046] Applying learner 'XGBoost' on task 'tarjetas' (iter 1/10) 
INFO  [14:13:28.380] Applying learner 'Bayes' on task 'tarjetas' (iter 5/10) 
prediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingvariables are collinearprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleadingprediction from a rank-deficient fit may be misleading
INFO  [14:21:08.923] Finished benchmark 
measures = list(
  msr("classif.auc", id = "auc_train", predict_sets = "train"),
  msr("classif.auc", id = "auc_test"),
  msr("classif.acc", id = "acc_train", predict_sets = "train"),
  msr("classif.acc", id = "acc_test"),
  msr("classif.fnr", id = "fnr_train", predict_sets = "train"),
  msr("classif.fnr", id = "fnr_test")
)

bmr$aggregate(measures)
NA

tab = bmr$aggregate(measures)

ranks = tab[, .(learner_id, rank_train = rank(-auc_train), rank_test = rank(-auc_test)), by = task_id]
print(ranks)
autoplot(bmr$clone(deep = TRUE), type = "roc")

autoplot(bmr, measure = msr("classif.auc")) + 
           theme(axis.text.x = element_text(angle = 45, hjust = 1))

NA
NA
autoplot(bmr, measure = msr("classif.acc")) + 
           theme(axis.text.x = element_text(angle = 45, hjust = 1))

NA
NA

LS0tCnRpdGxlOiAiQmVuY2htYXJrIGRlIG1vZGVsbyIKYXV0aG9yOiAiTWlndWVsIENvdG8gR2FyY2lhIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpDYXJnYW5kbyBwYXF1ZXRlcwpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQoKbGlicmFyeShoZXJlKQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShtbHIzKQpsaWJyYXJ5KCJtbHIzdml6IikKbGlicmFyeSgibWxyM2xlYXJuZXJzIikKbGlicmFyeShtbHIzcGlwZWxpbmVzKQpgYGAKCiMgUnV0YXMKYGBge3J9CgpyYXdfZGF0YSA8LSBoZXJlKCJkYXRhIiwgInJhdyIpCmludGVyaW1fZGF0YSA8LSBoZXJlKCJkYXRhIiwgImludGVyaW0iKQpwcm9jZXNzZWRfZGF0YSA8LSBoZXJlKCJkYXRhIiwgInByb2Nlc3NlZCIpCgpgYGAKCiMjIExleWVuZG8gYmFzZXMKYGBge3J9CmJhc2UgPC0gcmVhZFJEUyhmaWxlPXBhc3RlMChwcm9jZXNzZWRfZGF0YSwiL2Jhc2VfZmluYWwuUmRzIikpCgpiYXNlIDwtIGJhc2UgJT4lCiAgbXV0YXRlKG9iamV0aXZvID0gaWZfZWxzZShvYmpldGl2bz09MCwgImJ1ZW5vIiwgIm1hbG8iKSkKCmhlYWQoYmFzZSkKYGBgCgojIFRhcmVhIGRlIGFwcmVuZGl6YWplCiMjIERlZmluaWNpb24KYGBge3J9CnRhc2tfdGFyaiA9IFRhc2tDbGFzc2lmJG5ldyhpZCA9ICJ0YXJqZXRhcyIsIGJhY2tlbmQgPSBiYXNlLCB0YXJnZXQgPSAib2JqZXRpdm8iKQpwcmludCh0YXNrX3RhcmopCmBgYAoKIyMgCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CmF1dG9wbG90KHRhc2tfdGFyaiRzZWxlY3QoYygic2FsZG9fdGFyamV0YSIsICJjb2VmaWNpZW50ZV9zb2x2ZW5jaWEiLCAiZWRhZCIsICJsaW1pdGVfdGFyamV0YV9jcmVkaXRvIikpLCB0eXBlID0gInBhaXJzIikKdGFza190YXJqID0gVGFza0NsYXNzaWYkbmV3KGlkID0gInRhcmpldGFzIiwgYmFja2VuZCA9IGJhc2UsIHRhcmdldCA9ICJidWVuX3BhZ2Fkb3IiKQpgYGAKCiMgTW9kZWxvcwoKIyMgTGlzdGEgZGUgbW9kZWxvcwoKYGBge3J9Cgpscm5fcnBhcnQgPSBscm4oImNsYXNzaWYucnBhcnQiLCBwcmVkaWN0X3R5cGUgPSAicHJvYiIsIHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKSkKbHJuX2dsbW5ldCA9IGxybigiY2xhc3NpZi5nbG1uZXQiLCBwcmVkaWN0X3R5cGUgPSAicHJvYiIsIHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKSkKbHJuX2tubiA9IGxybigiY2xhc3NpZi5ra25uIiwgcHJlZGljdF90eXBlID0gInByb2IiLCBwcmVkaWN0X3NldHMgPSBjKCJ0cmFpbiIsICJ0ZXN0IikpCmxybl9sZGEgPSBscm4oImNsYXNzaWYubGRhIiwgcHJlZGljdF90eXBlID0gInByb2IiLCBwcmVkaWN0X3NldHMgPSBjKCJ0cmFpbiIsICJ0ZXN0IikpCmxybl9sb2dfcmVnID0gbHJuKCJjbGFzc2lmLmxvZ19yZWciLCBwcmVkaWN0X3R5cGUgPSAicHJvYiIsIHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKSkKbHJuX25uZXQgPSBscm4oImNsYXNzaWYubm5ldCIsIHByZWRpY3RfdHlwZSA9ICJwcm9iIiwgcHJlZGljdF9zZXRzID0gYygidHJhaW4iLCAidGVzdCIpKQpscm5fcmYgPSBscm4oImNsYXNzaWYucmFuZ2VyIiwgcHJlZGljdF90eXBlID0gInByb2IiLCBwcmVkaWN0X3NldHMgPSBjKCJ0cmFpbiIsICJ0ZXN0IikpCmxybl9zdm0gPSBscm4oImNsYXNzaWYuc3ZtIiwgcHJlZGljdF90eXBlID0gInByb2IiLCBwcmVkaWN0X3NldHMgPSBjKCJ0cmFpbiIsICJ0ZXN0IikpCmxybl94Z2Jvb3N0ID0gbHJuKCJjbGFzc2lmLnhnYm9vc3QiLCBwcmVkaWN0X3R5cGUgPSAicHJvYiIsIHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKSkKCmBgYAoKIyMgU2VwYXJhY2lvbiBkZSB0cmFpbmluZyB5IHRlc3RpbmcKYGBge3J9CnNldC5zZWVkKDM0Njc4KQp0cmFpbl9zZXQgPSBzYW1wbGUodGFza190YXJqJG5yb3csIDAuOCAqIHRhc2tfdGFyaiRucm93KQp0ZXN0X3NldCA9IHNldGRpZmYoc2VxX2xlbih0YXNrX3RhcmokbnJvdyksIHRyYWluX3NldCkKYGBgCgoKIyBCZW5jaG1hcmsKCiMjIFByZXByYWNpb24gZGUgZGF0b3MgcGFyYSBlbCBiZW5jaG1hcmsKYGBge3J9CnRoID0gCmltcHV0ZV9mY3RzIDwtIHBvKCJpbXB1dGVtb2RlIiwgYWZmZWN0X2NvbHVtbnMgPSBzZWxlY3Rvcl90eXBlKCJmYWN0b3IiKSkKaW1wdXRlX251bXMgPC0gcG8oImltcHV0ZWhpc3QiLCBhZmZlY3RfY29sdW1ucyA9IHNlbGVjdG9yX3R5cGUoIm51bWVyaWMiKSkKZW5jb2RlIDwtIHBvKCJlbmNvZGUiLCBhZmZlY3RfY29sdW1ucyA9IHNlbGVjdG9yX3R5cGUoImZhY3RvciIpKQp0aHJlc2hvbGQgPC0gcG8oInRocmVzaG9sZCIsIHBhcmFtX3ZhbHMgPSBsaXN0KHRocmVzaG9sZHMgPSAwLjkpKQoKcHJlX3Byb2Nlc2FtaWVudG8gPC0gaW1wdXRlX2ZjdHMgJT4+JSAKICBpbXB1dGVfbnVtcyAlPj4lIAogIGVuY29kZQoKbHJuX3JwYXJ0IDwtIEdyYXBoTGVhcm5lciRuZXcocHJlX3Byb2Nlc2FtaWVudG8gJT4+JSBwbyhscm5fcnBhcnQpKQpscm5fZ2xtbmV0IDwtIEdyYXBoTGVhcm5lciRuZXcocHJlX3Byb2Nlc2FtaWVudG8gJT4+JSBwbyhscm5fZ2xtbmV0KSkKbHJuX2tubiA8LSBHcmFwaExlYXJuZXIkbmV3KHByZV9wcm9jZXNhbWllbnRvICU+PiUgcG8obHJuX2tubikpCmxybl9sZGEgPC0gR3JhcGhMZWFybmVyJG5ldyhwcmVfcHJvY2VzYW1pZW50byAlPj4lIHBvKGxybl9sZGEpKQpscm5fbG9nX3JlZyA8LSBHcmFwaExlYXJuZXIkbmV3KHByZV9wcm9jZXNhbWllbnRvICU+PiUgcG8obHJuX2xvZ19yZWcpKQpscm5fbm5ldCA8LSBHcmFwaExlYXJuZXIkbmV3KHByZV9wcm9jZXNhbWllbnRvICU+PiUgcG8obHJuX25uZXQpKQpscm5fcmYgPC0gR3JhcGhMZWFybmVyJG5ldyhwcmVfcHJvY2VzYW1pZW50byAlPj4lIHBvKGxybl9yZikpCmxybl9zdm0gPC0gR3JhcGhMZWFybmVyJG5ldyhwcmVfcHJvY2VzYW1pZW50byAlPj4lIHBvKGxybl9zdm0pKQpscm5feGdib29zdCA8LSBHcmFwaExlYXJuZXIkbmV3KHByZV9wcm9jZXNhbWllbnRvICU+PiUgcG8obHJuX3hnYm9vc3QpKQoKbHJuX3JwYXJ0JHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKQpscm5fZ2xtbmV0JHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKQpscm5fa25uJHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKQpscm5fbGRhJHByZWRpY3Rfc2V0cyA9IGMoInRyYWluIiwgInRlc3QiKQpscm5fbG9nX3JlZyRwcmVkaWN0X3NldHMgPSBjKCJ0cmFpbiIsICJ0ZXN0IikKbHJuX25uZXQkcHJlZGljdF9zZXRzID0gYygidHJhaW4iLCAidGVzdCIpCmxybl9yZiRwcmVkaWN0X3NldHMgPSBjKCJ0cmFpbiIsICJ0ZXN0IikKbHJuX3N2bSRwcmVkaWN0X3NldHMgPSBjKCJ0cmFpbiIsICJ0ZXN0IikKbHJuX3hnYm9vc3QkcHJlZGljdF9zZXRzID0gYygidHJhaW4iLCAidGVzdCIpCgpscm5fcnBhcnQkaWQgPSAiw4FyYm9sIgpscm5fZ2xtbmV0JGlkID0gIlJlZy1yZWciCmxybl9rbm4kaWQgPSAiSy12ZWNpbm9zIgpscm5fbGRhJGlkID0gIkxEQSIKbHJuX2xvZ19yZWckaWQgPSAiUmVnLWxvZyIKbHJuX25uZXQkaWQgPSAiUmVkLU5ldXIiCmxybl9yZiRpZCA9ICJSYW5kb21Gb3Jlc3QiCmxybl9zdm0kaWQgPSAiU1ZNIgpscm5feGdib29zdCRpZCA9ICJYR0Jvb3N0IgoKbGVhcm5lcnMgPSBsaXN0KAogIGxybl9ycGFydCwKICBscm5fZ2xtbmV0LAogIGxybl9rbm4sCiAgbHJuX2xkYSwKICBscm5fbG9nX3JlZywKICAjbHJuX2JheWVzLAogIGxybl9yZiwKICBscm5fc3ZtLAogIGxybl94Z2Jvb3N0CikKCgpgYGAKCiMjIERlZmluY2lvbiBkZWwgYmVuY2htYXJrCmBgYHtyfQpyZXNhbXBsaW5ncyA9IHJzbXAoImN2IiwgZm9sZHMgPSAxMCkKCmRlc2lnbiA9IGJlbmNobWFya19ncmlkKHRhc2tfdGFyaiwgbGVhcm5lcnMsIHJlc2FtcGxpbmdzKQoKYGBgCgoKIyMgRWplY3VjaW9uCmBgYHtyfQpmdXR1cmU6OnBsYW4oIm11bHRpcHJvY2VzcyIpCgpibXIgPSBiZW5jaG1hcmsoZGVzaWduKQpgYGAKCgoKYGBge3J9Cm1lYXN1cmVzID0gbGlzdCgKICBtc3IoImNsYXNzaWYuYXVjIiwgaWQgPSAiYXVjX3RyYWluIiwgcHJlZGljdF9zZXRzID0gInRyYWluIiksCiAgbXNyKCJjbGFzc2lmLmF1YyIsIGlkID0gImF1Y190ZXN0IiksCiAgbXNyKCJjbGFzc2lmLmFjYyIsIGlkID0gImFjY190cmFpbiIsIHByZWRpY3Rfc2V0cyA9ICJ0cmFpbiIpLAogIG1zcigiY2xhc3NpZi5hY2MiLCBpZCA9ICJhY2NfdGVzdCIpLAogIG1zcigiY2xhc3NpZi5mcHIiLCBpZCA9ICJmcHJfdHJhaW4iLCBwcmVkaWN0X3NldHMgPSAidHJhaW4iKSwKICBtc3IoImNsYXNzaWYuZnByIiwgaWQgPSAiZnByX3Rlc3QiKQopCgpibXIkYWdncmVnYXRlKG1lYXN1cmVzKQoKYGBgCgoKYGBge3J9Cgp0YWIgPSBibXIkYWdncmVnYXRlKG1lYXN1cmVzKQoKcmFua3MgPSB0YWJbLCAuKGxlYXJuZXJfaWQsIHJhbmtfdHJhaW4gPSByYW5rKC1hdWNfdHJhaW4pLCByYW5rX3Rlc3QgPSByYW5rKC1hdWNfdGVzdCkpLCBieSA9IHRhc2tfaWRdCnByaW50KHJhbmtzKQpgYGAKCgpgYGB7cn0KYXV0b3Bsb3QoYm1yJGNsb25lKGRlZXAgPSBUUlVFKSwgdHlwZSA9ICJyb2MiKQoKYGBgCgoKCmBgYHtyfQphdXRvcGxvdChibXIsIG1lYXN1cmUgPSBtc3IoImNsYXNzaWYuYXVjIikpICsgCiAgICAgICAgICAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgaGp1c3QgPSAxKSkKCgpgYGAKCmBgYHtyfQphdXRvcGxvdChibXIsIG1lYXN1cmUgPSBtc3IoImNsYXNzaWYuYWNjIikpICsgCiAgICAgICAgICAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgaGp1c3QgPSAxKSkKCgpgYGAKCmBgYHtyfQphdXRvcGxvdChibXIsIG1lYXN1cmUgPSBtc3IoImNsYXNzaWYuZnByIikpICsgCiAgICAgICAgICAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgaGp1c3QgPSAxKSkKCgpgYGAKCgo=